{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multinomial AUC/PR AUC demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 . connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O_cluster_uptime:</td>\n",
       "<td>38 mins 14 secs</td></tr>\n",
       "<tr><td>H2O_cluster_timezone:</td>\n",
       "<td>Europe/Berlin</td></tr>\n",
       "<tr><td>H2O_data_parsing_timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O_cluster_version:</td>\n",
       "<td>3.33.0.99999</td></tr>\n",
       "<tr><td>H2O_cluster_version_age:</td>\n",
       "<td>47 minutes </td></tr>\n",
       "<tr><td>H2O_cluster_name:</td>\n",
       "<td>mori</td></tr>\n",
       "<tr><td>H2O_cluster_total_nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O_cluster_free_memory:</td>\n",
       "<td>4.851 Gb</td></tr>\n",
       "<tr><td>H2O_cluster_total_cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O_cluster_allowed_cores:</td>\n",
       "<td>8</td></tr>\n",
       "<tr><td>H2O_cluster_status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O_connection_url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O_connection_proxy:</td>\n",
       "<td>{\"http\": null, \"https\": null}</td></tr>\n",
       "<tr><td>H2O_internal_security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O_API_Extensions:</td>\n",
       "<td>Algos, Core V3, Core V4</td></tr>\n",
       "<tr><td>Python_version:</td>\n",
       "<td>3.7.3 candidate</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  -----------------------------\n",
       "H2O_cluster_uptime:         38 mins 14 secs\n",
       "H2O_cluster_timezone:       Europe/Berlin\n",
       "H2O_data_parsing_timezone:  UTC\n",
       "H2O_cluster_version:        3.33.0.99999\n",
       "H2O_cluster_version_age:    47 minutes\n",
       "H2O_cluster_name:           mori\n",
       "H2O_cluster_total_nodes:    1\n",
       "H2O_cluster_free_memory:    4.851 Gb\n",
       "H2O_cluster_total_cores:    8\n",
       "H2O_cluster_allowed_cores:  8\n",
       "H2O_cluster_status:         locked, healthy\n",
       "H2O_connection_url:         http://localhost:54321\n",
       "H2O_connection_proxy:       {\"http\": null, \"https\": null}\n",
       "H2O_internal_security:      False\n",
       "H2O_API_Extensions:         Algos, Core V3, Core V4\n",
       "Python_version:             3.7.3 candidate\n",
       "--------------------------  -----------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import h2o\n",
    "from h2o.estimators import H2OGradientBoostingEstimator\n",
    "from h2o.utils.shared_utils import _locate # private function - used to find files within h2o git project directory\n",
    "\n",
    "h2o.init(strict_version_check=False, port=54321)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "Model Details\n",
      "=============\n",
      "H2OGradientBoostingEstimator :  Gradient Boosting Machine\n",
      "Model Key:  GBM_model_python_1606223609474_67\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>100.0</td>\n",
       "      <td>500.0</td>\n",
       "      <td>67451.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.994</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.078</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0              100.0                     500.0              67451.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0        1.0        3.0       2.994         2.0         8.0        6.078  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsMultinomial: gbm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.07963805354633585\n",
      "RMSE: 0.28220215014477806\n",
      "LogLoss: 0.32402399831181045\n",
      "Mean Per-Class Error: 0.4120046082949308\n",
      "AUC: 0.999511857555245\n",
      "AUCPR: 0.998951571914844\n",
      "\n",
      "Multinomial AUC values: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>first_class_domain</th>\n",
       "      <th>second_class_domain</th>\n",
       "      <th>auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 vs Rest</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4 vs Rest</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5 vs Rest</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 vs Rest</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8 vs Rest</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Macro OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Weighted OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Class 3 vs. 4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.994624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Class 3 vs. 5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Class 3 vs. 6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Class 3 vs. 8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Class 4 vs. 5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.975269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Class 4 vs. 6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.999482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Class 4 vs. 8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Class 5 vs. 6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.988095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Class 5 vs. 8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Class 6 vs. 8</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.999752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Macro OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Weighted OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995155</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type first_class_domain second_class_domain       auc\n",
       "0       3 vs Rest                  3                None  1.000000\n",
       "1       4 vs Rest                  4                None  0.998556\n",
       "2       5 vs Rest                  5                None  1.000000\n",
       "3       6 vs Rest                  6                None  0.999004\n",
       "4       8 vs Rest                  8                None  1.000000\n",
       "5       Macro OVR               None                None  0.999512\n",
       "6    Weighted OVR               None                None  0.999032\n",
       "7   Class 3 vs. 4                  3                   4  0.994624\n",
       "8   Class 3 vs. 5                  3                   5  1.000000\n",
       "9   Class 3 vs. 6                  3                   6  1.000000\n",
       "10  Class 3 vs. 8                  3                   8  1.000000\n",
       "11  Class 4 vs. 5                  4                   5  0.975269\n",
       "12  Class 4 vs. 6                  4                   6  0.999482\n",
       "13  Class 4 vs. 8                  4                   8  1.000000\n",
       "14  Class 5 vs. 6                  5                   6  0.988095\n",
       "15  Class 5 vs. 8                  5                   8  1.000000\n",
       "16  Class 6 vs. 8                  6                   8  0.999752\n",
       "17      Macro OVO               None                None  0.995722\n",
       "18   Weighted OVO               None                None  0.995155"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Multinomial auc_pr values: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>first_class_domain</th>\n",
       "      <th>second_class_domain</th>\n",
       "      <th>auc_pr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 vs Rest</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4 vs Rest</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5 vs Rest</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 vs Rest</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8 vs Rest</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Macro OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Weighted OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Class 3 vs. 4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.999897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Class 3 vs. 5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Class 3 vs. 6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Class 3 vs. 8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Class 4 vs. 5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.999489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Class 4 vs. 6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.998871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Class 4 vs. 8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Class 5 vs. 6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.999350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Class 5 vs. 8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Class 6 vs. 8</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.999689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Macro OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Weighted OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999642</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type first_class_domain second_class_domain    auc_pr\n",
       "0       3 vs Rest                  3                None  1.000000\n",
       "1       4 vs Rest                  4                None  0.998698\n",
       "2       5 vs Rest                  5                None  1.000000\n",
       "3       6 vs Rest                  6                None  0.996059\n",
       "4       8 vs Rest                  8                None  1.000000\n",
       "5       Macro OVR               None                None  0.998952\n",
       "6    Weighted OVR               None                None  0.998538\n",
       "7   Class 3 vs. 4                  3                   4  0.999897\n",
       "8   Class 3 vs. 5                  3                   5  1.000000\n",
       "9   Class 3 vs. 6                  3                   6  1.000000\n",
       "10  Class 3 vs. 8                  3                   8  1.000000\n",
       "11  Class 4 vs. 5                  4                   5  0.999489\n",
       "12  Class 4 vs. 6                  4                   6  0.998871\n",
       "13  Class 4 vs. 8                  4                   8  1.000000\n",
       "14  Class 5 vs. 6                  5                   6  0.999350\n",
       "15  Class 5 vs. 8                  5                   8  1.000000\n",
       "16  Class 6 vs. 8                  6                   8  0.999689\n",
       "17      Macro OVO               None                None  0.999730\n",
       "18   Weighted OVO               None                None  0.999642"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Confusion Matrix: Row labels: Actual class; Column labels: Predicted class\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>8</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3 / 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006452</td>\n",
       "      <td>1 / 155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3 / 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.053571</td>\n",
       "      <td>3 / 56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0 / 72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>10 / 289</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     3      4    5     6     8     Error      Rate\n",
       "0  0.0    3.0  0.0   0.0   0.0  1.000000     3 / 3\n",
       "1  0.0  154.0  0.0   1.0   0.0  0.006452   1 / 155\n",
       "2  0.0    2.0  0.0   1.0   0.0  1.000000     3 / 3\n",
       "3  0.0    2.0  0.0  53.0   1.0  0.053571    3 / 56\n",
       "4  0.0    0.0  0.0   0.0  72.0  0.000000    0 / 72\n",
       "5  0.0  161.0  0.0  55.0  73.0  0.034602  10 / 289"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Top-5 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>hit_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.965398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   k  hit_ratio\n",
       "0  1   0.965398\n",
       "1  2   1.000000\n",
       "2  3   1.000000\n",
       "3  4   1.000000\n",
       "4  5   1.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ModelMetricsMultinomial: gbm\n",
      "** Reported on validation data. **\n",
      "\n",
      "MSE: 0.08366695440378827\n",
      "RMSE: 0.2892524060466711\n",
      "LogLoss: 0.3345387893847581\n",
      "Mean Per-Class Error: 0.2181318681318681\n",
      "AUC: 0.7987492282997901\n",
      "AUCPR: 0.7975377394397984\n",
      "\n",
      "Multinomial AUC values: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>first_class_domain</th>\n",
       "      <th>second_class_domain</th>\n",
       "      <th>auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 vs Rest</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4 vs Rest</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0.996154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5 vs Rest</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 vs Rest</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8 vs Rest</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Macro OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.798749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Weighted OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.997714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Class 3 vs. 4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.990385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Class 3 vs. 5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Class 3 vs. 6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Class 3 vs. 8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Class 4 vs. 5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Class 4 vs. 6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.995192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Class 4 vs. 8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Class 5 vs. 6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Class 5 vs. 8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Class 6 vs. 8</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.998016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Macro OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.798359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Weighted OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.872818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type first_class_domain second_class_domain       auc\n",
       "0       3 vs Rest                  3                None  1.000000\n",
       "1       4 vs Rest                  4                None  0.996154\n",
       "2       5 vs Rest                  5                None  0.000000\n",
       "3       6 vs Rest                  6                None  0.997592\n",
       "4       8 vs Rest                  8                None  1.000000\n",
       "5       Macro OVR               None                None  0.798749\n",
       "6    Weighted OVR               None                None  0.997714\n",
       "7   Class 3 vs. 4                  3                   4  0.990385\n",
       "8   Class 3 vs. 5                  3                   5  0.500000\n",
       "9   Class 3 vs. 6                  3                   6  1.000000\n",
       "10  Class 3 vs. 8                  3                   8  1.000000\n",
       "11  Class 4 vs. 5                  4                   5  0.500000\n",
       "12  Class 4 vs. 6                  4                   6  0.995192\n",
       "13  Class 4 vs. 8                  4                   8  1.000000\n",
       "14  Class 5 vs. 6                  5                   6  0.500000\n",
       "15  Class 5 vs. 8                  5                   8  0.500000\n",
       "16  Class 6 vs. 8                  6                   8  0.998016\n",
       "17      Macro OVO               None                None  0.798359\n",
       "18   Weighted OVO               None                None  0.872818"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Multinomial auc_pr values: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>first_class_domain</th>\n",
       "      <th>second_class_domain</th>\n",
       "      <th>auc_pr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 vs Rest</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4 vs Rest</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0.994834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5 vs Rest</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 vs Rest</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>0.992854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8 vs Rest</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Macro OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.797538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Weighted OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Class 3 vs. 4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.999817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Class 3 vs. 5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Class 3 vs. 6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Class 3 vs. 8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Class 4 vs. 5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Class 4 vs. 6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.996311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Class 4 vs. 8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Class 5 vs. 6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Class 5 vs. 8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Class 6 vs. 8</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.997536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Macro OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.799366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Weighted OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.874012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type first_class_domain second_class_domain    auc_pr\n",
       "0       3 vs Rest                  3                None  1.000000\n",
       "1       4 vs Rest                  4                None  0.994834\n",
       "2       5 vs Rest                  5                None  0.000000\n",
       "3       6 vs Rest                  6                None  0.992854\n",
       "4       8 vs Rest                  8                None  1.000000\n",
       "5       Macro OVR               None                None  0.797538\n",
       "6    Weighted OVR               None                None  0.995994\n",
       "7   Class 3 vs. 4                  3                   4  0.999817\n",
       "8   Class 3 vs. 5                  3                   5  0.500000\n",
       "9   Class 3 vs. 6                  3                   6  1.000000\n",
       "10  Class 3 vs. 8                  3                   8  1.000000\n",
       "11  Class 4 vs. 5                  4                   5  0.500000\n",
       "12  Class 4 vs. 6                  4                   6  0.996311\n",
       "13  Class 4 vs. 8                  4                   8  1.000000\n",
       "14  Class 5 vs. 6                  5                   6  0.500000\n",
       "15  Class 5 vs. 8                  5                   8  0.500000\n",
       "16  Class 6 vs. 8                  6                   8  0.997536\n",
       "17      Macro OVO               None                None  0.799366\n",
       "18   Weighted OVO               None                None  0.874012"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Confusion Matrix: Row labels: Actual class; Column labels: Predicted class\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>8</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1 / 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>51.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>1 / 52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0 / 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>2 / 28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0 / 36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>4 / 117</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     3     4    5     6     8     Error     Rate\n",
       "0  0.0   1.0  0.0   0.0   0.0  1.000000    1 / 1\n",
       "1  0.0  51.0  0.0   1.0   0.0  0.019231   1 / 52\n",
       "2  0.0   0.0  0.0   0.0   0.0       NaN    0 / 0\n",
       "3  0.0   2.0  0.0  26.0   0.0  0.071429   2 / 28\n",
       "4  0.0   0.0  0.0   0.0  36.0  0.000000   0 / 36\n",
       "5  0.0  54.0  0.0  27.0  36.0  0.034188  4 / 117"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Top-5 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>hit_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.965812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   k  hit_ratio\n",
       "0  1   0.965812\n",
       "1  2   1.000000\n",
       "2  3   1.000000\n",
       "3  4   1.000000\n",
       "4  5   1.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_classification_error</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>validation_rmse</th>\n",
       "      <th>validation_logloss</th>\n",
       "      <th>validation_classification_error</th>\n",
       "      <th>validation_auc</th>\n",
       "      <th>validation_pr_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.001 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>0.643599</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.007 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.792456</td>\n",
       "      <td>1.572433</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.996920</td>\n",
       "      <td>0.923202</td>\n",
       "      <td>0.792423</td>\n",
       "      <td>1.572271</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798394</td>\n",
       "      <td>0.797055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.010 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.784944</td>\n",
       "      <td>1.536933</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998389</td>\n",
       "      <td>0.953458</td>\n",
       "      <td>0.784923</td>\n",
       "      <td>1.536834</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.013 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.777463</td>\n",
       "      <td>1.502819</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998044</td>\n",
       "      <td>0.945566</td>\n",
       "      <td>0.777452</td>\n",
       "      <td>1.502770</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.016 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.770014</td>\n",
       "      <td>1.470001</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998044</td>\n",
       "      <td>0.945566</td>\n",
       "      <td>0.769983</td>\n",
       "      <td>1.469861</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.019 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.762600</td>\n",
       "      <td>1.438399</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998054</td>\n",
       "      <td>0.945574</td>\n",
       "      <td>0.762554</td>\n",
       "      <td>1.438200</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.021 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.755219</td>\n",
       "      <td>1.407921</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998092</td>\n",
       "      <td>0.945606</td>\n",
       "      <td>0.755176</td>\n",
       "      <td>1.407739</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.024 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.747874</td>\n",
       "      <td>1.378506</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998092</td>\n",
       "      <td>0.945606</td>\n",
       "      <td>0.747814</td>\n",
       "      <td>1.378260</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.026 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.740564</td>\n",
       "      <td>1.350089</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998092</td>\n",
       "      <td>0.945606</td>\n",
       "      <td>0.740519</td>\n",
       "      <td>1.349903</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.028 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.733293</td>\n",
       "      <td>1.322618</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998092</td>\n",
       "      <td>0.945606</td>\n",
       "      <td>0.733238</td>\n",
       "      <td>1.322396</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.031 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.726058</td>\n",
       "      <td>1.296014</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998102</td>\n",
       "      <td>0.945614</td>\n",
       "      <td>0.726020</td>\n",
       "      <td>1.295866</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.034 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.718861</td>\n",
       "      <td>1.270251</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998102</td>\n",
       "      <td>0.945614</td>\n",
       "      <td>0.718823</td>\n",
       "      <td>1.270106</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.036 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.711704</td>\n",
       "      <td>1.245292</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998102</td>\n",
       "      <td>0.945614</td>\n",
       "      <td>0.711680</td>\n",
       "      <td>1.245194</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.038 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.704588</td>\n",
       "      <td>1.221078</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998102</td>\n",
       "      <td>0.945614</td>\n",
       "      <td>0.704565</td>\n",
       "      <td>1.220990</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.041 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.697512</td>\n",
       "      <td>1.197577</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.999277</td>\n",
       "      <td>0.998774</td>\n",
       "      <td>0.697507</td>\n",
       "      <td>1.197554</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.045 sec</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.690478</td>\n",
       "      <td>1.174763</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.999292</td>\n",
       "      <td>0.998837</td>\n",
       "      <td>0.690506</td>\n",
       "      <td>1.174846</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.048 sec</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.683489</td>\n",
       "      <td>1.152605</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.999292</td>\n",
       "      <td>0.998837</td>\n",
       "      <td>0.683503</td>\n",
       "      <td>1.152644</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.052 sec</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.676542</td>\n",
       "      <td>1.131059</td>\n",
       "      <td>0.031142</td>\n",
       "      <td>0.999311</td>\n",
       "      <td>0.998856</td>\n",
       "      <td>0.676589</td>\n",
       "      <td>1.131201</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.055 sec</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.669640</td>\n",
       "      <td>1.110108</td>\n",
       "      <td>0.031142</td>\n",
       "      <td>0.999311</td>\n",
       "      <td>0.998856</td>\n",
       "      <td>0.669721</td>\n",
       "      <td>1.110354</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.058 sec</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.662783</td>\n",
       "      <td>1.089722</td>\n",
       "      <td>0.031142</td>\n",
       "      <td>0.999311</td>\n",
       "      <td>0.998856</td>\n",
       "      <td>0.662885</td>\n",
       "      <td>1.090031</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.798834</td>\n",
       "      <td>0.798153</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2020-11-24 14:51:44   0.001 sec              0.0       0.800000   \n",
       "1     2020-11-24 14:51:44   0.007 sec              1.0       0.792456   \n",
       "2     2020-11-24 14:51:44   0.010 sec              2.0       0.784944   \n",
       "3     2020-11-24 14:51:44   0.013 sec              3.0       0.777463   \n",
       "4     2020-11-24 14:51:44   0.016 sec              4.0       0.770014   \n",
       "5     2020-11-24 14:51:44   0.019 sec              5.0       0.762600   \n",
       "6     2020-11-24 14:51:44   0.021 sec              6.0       0.755219   \n",
       "7     2020-11-24 14:51:44   0.024 sec              7.0       0.747874   \n",
       "8     2020-11-24 14:51:44   0.026 sec              8.0       0.740564   \n",
       "9     2020-11-24 14:51:44   0.028 sec              9.0       0.733293   \n",
       "10    2020-11-24 14:51:44   0.031 sec             10.0       0.726058   \n",
       "11    2020-11-24 14:51:44   0.034 sec             11.0       0.718861   \n",
       "12    2020-11-24 14:51:44   0.036 sec             12.0       0.711704   \n",
       "13    2020-11-24 14:51:44   0.038 sec             13.0       0.704588   \n",
       "14    2020-11-24 14:51:44   0.041 sec             14.0       0.697512   \n",
       "15    2020-11-24 14:51:44   0.045 sec             15.0       0.690478   \n",
       "16    2020-11-24 14:51:44   0.048 sec             16.0       0.683489   \n",
       "17    2020-11-24 14:51:44   0.052 sec             17.0       0.676542   \n",
       "18    2020-11-24 14:51:44   0.055 sec             18.0       0.669640   \n",
       "19    2020-11-24 14:51:44   0.058 sec             19.0       0.662783   \n",
       "\n",
       "    training_logloss  training_classification_error  training_auc  \\\n",
       "0           1.609438                       0.643599      0.500000   \n",
       "1           1.572433                       0.034602      0.996920   \n",
       "2           1.536933                       0.034602      0.998389   \n",
       "3           1.502819                       0.034602      0.998044   \n",
       "4           1.470001                       0.034602      0.998044   \n",
       "5           1.438399                       0.034602      0.998054   \n",
       "6           1.407921                       0.034602      0.998092   \n",
       "7           1.378506                       0.034602      0.998092   \n",
       "8           1.350089                       0.034602      0.998092   \n",
       "9           1.322618                       0.034602      0.998092   \n",
       "10          1.296014                       0.034602      0.998102   \n",
       "11          1.270251                       0.034602      0.998102   \n",
       "12          1.245292                       0.034602      0.998102   \n",
       "13          1.221078                       0.034602      0.998102   \n",
       "14          1.197577                       0.034602      0.999277   \n",
       "15          1.174763                       0.034602      0.999292   \n",
       "16          1.152605                       0.034602      0.999292   \n",
       "17          1.131059                       0.031142      0.999311   \n",
       "18          1.110108                       0.031142      0.999311   \n",
       "19          1.089722                       0.031142      0.999311   \n",
       "\n",
       "    training_pr_auc  validation_rmse  validation_logloss  \\\n",
       "0          0.200000         0.800000            1.609438   \n",
       "1          0.923202         0.792423            1.572271   \n",
       "2          0.953458         0.784923            1.536834   \n",
       "3          0.945566         0.777452            1.502770   \n",
       "4          0.945566         0.769983            1.469861   \n",
       "5          0.945574         0.762554            1.438200   \n",
       "6          0.945606         0.755176            1.407739   \n",
       "7          0.945606         0.747814            1.378260   \n",
       "8          0.945606         0.740519            1.349903   \n",
       "9          0.945606         0.733238            1.322396   \n",
       "10         0.945614         0.726020            1.295866   \n",
       "11         0.945614         0.718823            1.270106   \n",
       "12         0.945614         0.711680            1.245194   \n",
       "13         0.945614         0.704565            1.220990   \n",
       "14         0.998774         0.697507            1.197554   \n",
       "15         0.998837         0.690506            1.174846   \n",
       "16         0.998837         0.683503            1.152644   \n",
       "17         0.998856         0.676589            1.131201   \n",
       "18         0.998856         0.669721            1.110354   \n",
       "19         0.998856         0.662885            1.090031   \n",
       "\n",
       "    validation_classification_error  validation_auc  validation_pr_auc  \n",
       "0                          0.615385        0.400000           0.200000  \n",
       "1                          0.034188        0.798394           0.797055  \n",
       "2                          0.034188        0.798834           0.798153  \n",
       "3                          0.034188        0.798834           0.798153  \n",
       "4                          0.034188        0.798834           0.798153  \n",
       "5                          0.034188        0.798834           0.798153  \n",
       "6                          0.034188        0.798834           0.798153  \n",
       "7                          0.034188        0.798834           0.798153  \n",
       "8                          0.034188        0.798834           0.798153  \n",
       "9                          0.034188        0.798834           0.798153  \n",
       "10                         0.034188        0.798834           0.798153  \n",
       "11                         0.034188        0.798834           0.798153  \n",
       "12                         0.034188        0.798834           0.798153  \n",
       "13                         0.034188        0.798834           0.798153  \n",
       "14                         0.034188        0.798834           0.798153  \n",
       "15                         0.034188        0.798834           0.798153  \n",
       "16                         0.034188        0.798834           0.798153  \n",
       "17                         0.034188        0.798834           0.798153  \n",
       "18                         0.034188        0.798834           0.798153  \n",
       "19                         0.034188        0.798834           0.798153  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>displacement</td>\n",
       "      <td>6259.065430</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>power</td>\n",
       "      <td>56.853493</td>\n",
       "      <td>0.009083</td>\n",
       "      <td>0.008901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>acceleration</td>\n",
       "      <td>31.531071</td>\n",
       "      <td>0.005038</td>\n",
       "      <td>0.004937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>weight</td>\n",
       "      <td>21.419443</td>\n",
       "      <td>0.003422</td>\n",
       "      <td>0.003354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>year</td>\n",
       "      <td>18.227747</td>\n",
       "      <td>0.002912</td>\n",
       "      <td>0.002854</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       variable  relative_importance  scaled_importance  percentage\n",
       "0  displacement          6259.065430           1.000000    0.979955\n",
       "1         power            56.853493           0.009083    0.008901\n",
       "2  acceleration            31.531071           0.005038    0.004937\n",
       "3        weight            21.419443           0.003422    0.003354\n",
       "4          year            18.227747           0.002912    0.002854"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cars = h2o.import_file(path=_locate(\"smalldata/junit/cars_20mpg.csv\"))\n",
    "\n",
    "features = [\"displacement\", \"power\", \"weight\", \"acceleration\", \"year\"]\n",
    "response = \"cylinders\"\n",
    "distribution = \"multinomial\"\n",
    "\n",
    "cars[response] = cars[response].asfactor()\n",
    "\n",
    "r = cars[0].runif()\n",
    "train = cars[r > .3]\n",
    "valid = cars[r <= .3]\n",
    "    \n",
    "# train model\n",
    "gbm = H2OGradientBoostingEstimator(distribution=\"multinomial\", \n",
    "                                   ntrees=100, \n",
    "                                   max_depth=3, \n",
    "                                   learn_rate=0.01, \n",
    "                                   auc_type=\"MACRO_OVR\")\n",
    "gbm.train(x               =features, \n",
    "          y               =response, \n",
    "          training_frame  =train,\n",
    "          validation_frame=valid)\n",
    "gbm.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.999511857555245"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm.auc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.998951571914844"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm.aucpr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Multinomial AUC values: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>first_class_domain</th>\n",
       "      <th>second_class_domain</th>\n",
       "      <th>auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 vs Rest</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4 vs Rest</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>0.998556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5 vs Rest</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 vs Rest</td>\n",
       "      <td>6</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8 vs Rest</td>\n",
       "      <td>8</td>\n",
       "      <td>None</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Macro OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Weighted OVR</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.999032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Class 3 vs. 4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.994624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Class 3 vs. 5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Class 3 vs. 6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Class 3 vs. 8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Class 4 vs. 5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.975269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Class 4 vs. 6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0.999482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Class 4 vs. 8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Class 5 vs. 6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.988095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Class 5 vs. 8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Class 6 vs. 8</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.999752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Macro OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Weighted OVO</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0.995155</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             type first_class_domain second_class_domain       auc\n",
       "0       3 vs Rest                  3                None  1.000000\n",
       "1       4 vs Rest                  4                None  0.998556\n",
       "2       5 vs Rest                  5                None  1.000000\n",
       "3       6 vs Rest                  6                None  0.999004\n",
       "4       8 vs Rest                  8                None  1.000000\n",
       "5       Macro OVR               None                None  0.999512\n",
       "6    Weighted OVR               None                None  0.999032\n",
       "7   Class 3 vs. 4                  3                   4  0.994624\n",
       "8   Class 3 vs. 5                  3                   5  1.000000\n",
       "9   Class 3 vs. 6                  3                   6  1.000000\n",
       "10  Class 3 vs. 8                  3                   8  1.000000\n",
       "11  Class 4 vs. 5                  4                   5  0.975269\n",
       "12  Class 4 vs. 6                  4                   6  0.999482\n",
       "13  Class 4 vs. 8                  4                   8  1.000000\n",
       "14  Class 5 vs. 6                  5                   6  0.988095\n",
       "15  Class 5 vs. 8                  5                   8  1.000000\n",
       "16  Class 6 vs. 8                  6                   8  0.999752\n",
       "17      Macro OVO               None                None  0.995722\n",
       "18   Weighted OVO               None                None  0.995155"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm.multinomial_auc_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_classification_error</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>validation_rmse</th>\n",
       "      <th>validation_logloss</th>\n",
       "      <th>validation_classification_error</th>\n",
       "      <th>validation_auc</th>\n",
       "      <th>validation_pr_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.003 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>0.643599</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.387483</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.349551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.016 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.792456</td>\n",
       "      <td>1.572433</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.997761</td>\n",
       "      <td>0.993284</td>\n",
       "      <td>0.792423</td>\n",
       "      <td>1.572271</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.996925</td>\n",
       "      <td>0.994890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.020 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.784944</td>\n",
       "      <td>1.536933</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998140</td>\n",
       "      <td>0.995454</td>\n",
       "      <td>0.784923</td>\n",
       "      <td>1.536834</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>0.996420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.024 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.777463</td>\n",
       "      <td>1.502819</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998134</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.777452</td>\n",
       "      <td>1.502770</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>0.996420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.026 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.770014</td>\n",
       "      <td>1.470001</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998134</td>\n",
       "      <td>0.995058</td>\n",
       "      <td>0.769983</td>\n",
       "      <td>1.469861</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>0.996420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.029 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.762600</td>\n",
       "      <td>1.438399</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998160</td>\n",
       "      <td>0.995082</td>\n",
       "      <td>0.762554</td>\n",
       "      <td>1.438200</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>0.996420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:44</td>\n",
       "      <td>0.031 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.755219</td>\n",
       "      <td>1.407921</td>\n",
       "      <td>0.034602</td>\n",
       "      <td>0.998264</td>\n",
       "      <td>0.995168</td>\n",
       "      <td>0.755176</td>\n",
       "      <td>1.407739</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>0.997573</td>\n",
       "      <td>0.996420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0    2020-11-24 14:51:44   0.003 sec              0.0       0.800000   \n",
       "1    2020-11-24 14:51:44   0.016 sec              1.0       0.792456   \n",
       "2    2020-11-24 14:51:44   0.020 sec              2.0       0.784944   \n",
       "3    2020-11-24 14:51:44   0.024 sec              3.0       0.777463   \n",
       "4    2020-11-24 14:51:44   0.026 sec              4.0       0.770014   \n",
       "5    2020-11-24 14:51:44   0.029 sec              5.0       0.762600   \n",
       "6    2020-11-24 14:51:44   0.031 sec              6.0       0.755219   \n",
       "\n",
       "   training_logloss  training_classification_error  training_auc  \\\n",
       "0          1.609438                       0.643599      0.500000   \n",
       "1          1.572433                       0.034602      0.997761   \n",
       "2          1.536933                       0.034602      0.998140   \n",
       "3          1.502819                       0.034602      0.998134   \n",
       "4          1.470001                       0.034602      0.998134   \n",
       "5          1.438399                       0.034602      0.998160   \n",
       "6          1.407921                       0.034602      0.998264   \n",
       "\n",
       "   training_pr_auc  validation_rmse  validation_logloss  \\\n",
       "0         0.387483         0.800000            1.609438   \n",
       "1         0.993284         0.792423            1.572271   \n",
       "2         0.995454         0.784923            1.536834   \n",
       "3         0.995058         0.777452            1.502770   \n",
       "4         0.995058         0.769983            1.469861   \n",
       "5         0.995082         0.762554            1.438200   \n",
       "6         0.995168         0.755176            1.407739   \n",
       "\n",
       "   validation_classification_error  validation_auc  validation_pr_auc  \n",
       "0                         0.615385        0.500000           0.349551  \n",
       "1                         0.034188        0.996925           0.994890  \n",
       "2                         0.034188        0.997573           0.996420  \n",
       "3                         0.034188        0.997573           0.996420  \n",
       "4                         0.034188        0.997573           0.996420  \n",
       "5                         0.034188        0.997573           0.996420  \n",
       "6                         0.034188        0.997573           0.996420  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# early stopping\n",
    "\n",
    "gbm = H2OGradientBoostingEstimator(distribution=\"multinomial\", \n",
    "                                   ntrees=100, \n",
    "                                   max_depth=3, \n",
    "                                   learn_rate=0.01,\n",
    "                                   stopping_metric=\"AUCPR\", \n",
    "                                   stopping_tolerance=0.01, \n",
    "                                   stopping_rounds=3,\n",
    "                                   auc_type=\"WEIGHTED_OVR\")\n",
    "gbm.train(x               =features, \n",
    "          y               =response, \n",
    "          training_frame  =train,\n",
    "          validation_frame=valid)\n",
    "gbm.scoring_history()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Grid Build progress: |████████████████████████████████████████████████| 100%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'Grid_GBM_py_3_sid_b9cb_model_python_1606223609474_69_model_3': 0.9996445050710137,\n",
       " 'Grid_GBM_py_3_sid_b9cb_model_python_1606223609474_69_model_4': 0.9996445050710137,\n",
       " 'Grid_GBM_py_3_sid_b9cb_model_python_1606223609474_69_model_2': 0.9994134025192564,\n",
       " 'Grid_GBM_py_3_sid_b9cb_model_python_1606223609474_69_model_1': 0.9994134025192564}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# grid search\n",
    "\n",
    "from h2o.grid.grid_search import H2OGridSearch\n",
    "hyper_parameters = {'ntrees': [5, 10], 'max_depth': [10, 20]}\n",
    "gs = H2OGridSearch(H2OGradientBoostingEstimator(distribution = \"multinomial\", auc_type=\"MACRO_OVR\"), hyper_parameters)\n",
    "gs.train(x=features, y=response, training_frame=train)\n",
    "gs.auc(train=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n",
      "Model Details\n",
      "=============\n",
      "H2OGradientBoostingEstimator :  Gradient Boosting Machine\n",
      "Model Key:  GBM_model_python_1606223609474_70\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>100.0</td>\n",
       "      <td>9800.0</td>\n",
       "      <td>2008108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.99949</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.804898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0              100.0                    9800.0            2008108.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0        1.0        3.0     2.99949         2.0         8.0     7.804898  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsMultinomial: gbm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.430057090190663\n",
      "RMSE: 0.6557873818477015\n",
      "LogLoss: 1.3717578431630804\n",
      "Mean Per-Class Error: 0.7578920698567744\n",
      "AUC: NaN\n",
      "AUCPR: NaN\n",
      "Multinomial auc values: Table is not computed because it is disabled or due to domain size (maximum is 50 domains).\n",
      "Multinomial auc_pr values: Table is not computed because it is disabled or due to domain size (maximum is 50 domains).\n",
      "\n",
      "Confusion Matrix: Row labels: Actual class; Column labels: Predicted class\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ABE</th>\n",
       "      <th>ABQ</th>\n",
       "      <th>ACY</th>\n",
       "      <th>ALB</th>\n",
       "      <th>ATL</th>\n",
       "      <th>AVP</th>\n",
       "      <th>BDL</th>\n",
       "      <th>BGM</th>\n",
       "      <th>BNA</th>\n",
       "      <th>BOS</th>\n",
       "      <th>...</th>\n",
       "      <th>SNA</th>\n",
       "      <th>STL</th>\n",
       "      <th>SWF</th>\n",
       "      <th>SYR</th>\n",
       "      <th>TOL</th>\n",
       "      <th>TPA</th>\n",
       "      <th>TUS</th>\n",
       "      <th>UCA</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>102 / 102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>76 / 76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>15 / 15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>81 / 81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18 / 18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>20 / 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>48 / 63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.794872</td>\n",
       "      <td>31 / 39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>41 / 41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>114.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.673352</td>\n",
       "      <td>235 / 349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>30 / 30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.859813</td>\n",
       "      <td>92 / 107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.658046</td>\n",
       "      <td>229 / 348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.591503</td>\n",
       "      <td>181 / 306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>21 / 21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>22 / 22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1 / 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0 / 13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40 / 40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.421769</td>\n",
       "      <td>62 / 147</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    ABE  ABQ  ACY  ALB  ATL  AVP   BDL  BGM  BNA    BOS  ...   SNA  STL  SWF  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "5   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "6   0.0  0.0  0.0  0.0  0.0  0.0  15.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "7   0.0  0.0  0.0  0.0  0.0  0.0  10.0  8.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "8   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "9   0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0  114.0  ...   0.0  0.0  0.0   \n",
       "10  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "11  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0   36.0  ...   0.0  0.0  0.0   \n",
       "12  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...  17.0  0.0  0.0   \n",
       "13  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    9.0  ...   0.0  0.0  0.0   \n",
       "14  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "15  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "16  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "17  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "18  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    0.0  ...   0.0  0.0  0.0   \n",
       "19  0.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0  0.0    4.0  ...   0.0  0.0  0.0   \n",
       "\n",
       "    SYR  TOL  TPA  TUS  UCA     Error       Rate  \n",
       "0   0.0  0.0  0.0  0.0  0.0  1.000000  102 / 102  \n",
       "1   0.0  0.0  0.0  0.0  0.0  1.000000    76 / 76  \n",
       "2   0.0  0.0  0.0  0.0  0.0  1.000000    15 / 15  \n",
       "3   0.0  0.0  0.0  0.0  0.0  1.000000    81 / 81  \n",
       "4   0.0  0.0  0.0  0.0  0.0  1.000000    18 / 18  \n",
       "5   0.0  0.0  0.0  0.0  0.0  1.000000    20 / 20  \n",
       "6   0.0  0.0  0.0  0.0  0.0  0.761905    48 / 63  \n",
       "7   0.0  0.0  0.0  0.0  0.0  0.794872    31 / 39  \n",
       "8   0.0  0.0  0.0  0.0  0.0  1.000000    41 / 41  \n",
       "9   0.0  0.0  0.0  0.0  0.0  0.673352  235 / 349  \n",
       "10  0.0  0.0  0.0  0.0  0.0  1.000000    30 / 30  \n",
       "11  0.0  0.0  0.0  0.0  8.0  0.859813   92 / 107  \n",
       "12  0.0  0.0  0.0  5.0  0.0  0.658046  229 / 348  \n",
       "13  0.0  0.0  0.0  0.0  0.0  0.591503  181 / 306  \n",
       "14  0.0  0.0  0.0  0.0  0.0  1.000000    21 / 21  \n",
       "15  0.0  0.0  0.0  0.0  0.0  1.000000    22 / 22  \n",
       "16  0.0  0.0  0.0  0.0  0.0  1.000000      1 / 1  \n",
       "17  0.0  0.0  0.0  0.0  0.0  0.000000     0 / 13  \n",
       "18  0.0  0.0  0.0  0.0  0.0  1.000000    40 / 40  \n",
       "19  0.0  0.0  0.0  0.0  0.0  0.421769   62 / 147  \n",
       "\n",
       "[20 rows x 100 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Top-10 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>hit_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.647989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.772855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.834294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.872992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.900168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.920155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.933615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.943201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.950798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0.956356</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k  hit_ratio\n",
       "0   1   0.647989\n",
       "1   2   0.772855\n",
       "2   3   0.834294\n",
       "3   4   0.872992\n",
       "4   5   0.900168\n",
       "5   6   0.920155\n",
       "6   7   0.933615\n",
       "7   8   0.943201\n",
       "8   9   0.950798\n",
       "9  10   0.956356"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ModelMetricsMultinomial: gbm\n",
      "** Reported on validation data. **\n",
      "\n",
      "MSE: 0.45785658129894297\n",
      "RMSE: 0.6766510040626135\n",
      "LogLoss: 1.6072449485890636\n",
      "Mean Per-Class Error: 0.794834751023698\n",
      "AUC: NaN\n",
      "AUCPR: NaN\n",
      "Multinomial auc values: Table is not computed because it is disabled or due to domain size (maximum is 50 domains).\n",
      "Multinomial auc_pr values: Table is not computed because it is disabled or due to domain size (maximum is 50 domains).\n",
      "\n",
      "Confusion Matrix: Row labels: Actual class; Column labels: Predicted class\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ABE</th>\n",
       "      <th>ABQ</th>\n",
       "      <th>ACY</th>\n",
       "      <th>ALB</th>\n",
       "      <th>ATL</th>\n",
       "      <th>AVP</th>\n",
       "      <th>BDL</th>\n",
       "      <th>BGM</th>\n",
       "      <th>BNA</th>\n",
       "      <th>BOS</th>\n",
       "      <th>...</th>\n",
       "      <th>SNA</th>\n",
       "      <th>STL</th>\n",
       "      <th>SWF</th>\n",
       "      <th>SYR</th>\n",
       "      <th>TOL</th>\n",
       "      <th>TPA</th>\n",
       "      <th>TUS</th>\n",
       "      <th>UCA</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28 / 28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>21 / 21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4 / 4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>17 / 17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5 / 5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7 / 7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18 / 18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>14 / 14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8 / 8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.712766</td>\n",
       "      <td>67 / 94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7 / 7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>19 / 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.740260</td>\n",
       "      <td>57 / 77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.712644</td>\n",
       "      <td>62 / 87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3 / 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3 / 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2 / 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0 / 6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>12 / 12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>16 / 36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    ABE  ABQ  ACY  ALB  ATL  AVP  BDL  BGM  BNA   BOS  ...  SNA  STL  SWF  \\\n",
       "0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "1   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "2   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "3   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "4   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "5   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "6   0.0  0.0  0.0  0.0  0.0  0.0  0.0  4.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "7   0.0  0.0  0.0  0.0  0.0  0.0  3.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "8   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "9   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  27.0  ...  0.0  0.0  0.0   \n",
       "10  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   1.0  ...  0.0  0.0  0.0   \n",
       "11  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   7.0  ...  0.0  0.0  0.0   \n",
       "12  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  3.0  0.0  0.0   \n",
       "13  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   3.0  ...  0.0  0.0  0.0   \n",
       "14  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "15  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "16  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "17  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "18  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   0.0  ...  0.0  0.0  0.0   \n",
       "19  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0   1.0  ...  0.0  0.0  0.0   \n",
       "\n",
       "    SYR  TOL  TPA  TUS  UCA     Error     Rate  \n",
       "0   0.0  0.0  0.0  0.0  0.0  1.000000  28 / 28  \n",
       "1   0.0  0.0  0.0  0.0  0.0  1.000000  21 / 21  \n",
       "2   0.0  0.0  0.0  0.0  0.0  1.000000    4 / 4  \n",
       "3   0.0  0.0  0.0  0.0  0.0  1.000000  17 / 17  \n",
       "4   0.0  0.0  0.0  0.0  0.0  1.000000    5 / 5  \n",
       "5   0.0  0.0  0.0  0.0  0.0  1.000000    7 / 7  \n",
       "6   0.0  0.0  0.0  0.0  0.0  1.000000  18 / 18  \n",
       "7   0.0  0.0  0.0  0.0  0.0  1.000000  14 / 14  \n",
       "8   0.0  0.0  0.0  0.0  0.0  1.000000    8 / 8  \n",
       "9   0.0  0.0  0.0  0.0  0.0  0.712766  67 / 94  \n",
       "10  0.0  0.0  0.0  0.0  0.0  1.000000    7 / 7  \n",
       "11  0.0  0.0  0.0  0.0  4.0  0.950000  19 / 20  \n",
       "12  0.0  0.0  0.0  1.0  0.0  0.740260  57 / 77  \n",
       "13  0.0  0.0  0.0  0.0  0.0  0.712644  62 / 87  \n",
       "14  0.0  0.0  0.0  0.0  0.0  1.000000    3 / 3  \n",
       "15  0.0  0.0  0.0  0.0  0.0  1.000000    3 / 3  \n",
       "16  0.0  0.0  0.0  0.0  0.0  1.000000    2 / 2  \n",
       "17  0.0  0.0  0.0  0.0  0.0  0.000000    0 / 6  \n",
       "18  0.0  0.0  0.0  0.0  0.0  1.000000  12 / 12  \n",
       "19  0.0  0.0  0.0  0.0  0.0  0.444444  16 / 36  \n",
       "\n",
       "[20 rows x 100 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Top-10 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>hit_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.611065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.737729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.778910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.814476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.844218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.864601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.878744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.891015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.900999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0.909526</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k  hit_ratio\n",
       "0   1   0.611065\n",
       "1   2   0.737729\n",
       "2   3   0.778910\n",
       "3   4   0.814476\n",
       "4   5   0.844218\n",
       "5   6   0.864601\n",
       "6   7   0.878744\n",
       "7   8   0.891015\n",
       "8   9   0.900999\n",
       "9  10   0.909526"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_classification_error</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>validation_rmse</th>\n",
       "      <th>validation_logloss</th>\n",
       "      <th>validation_classification_error</th>\n",
       "      <th>validation_auc</th>\n",
       "      <th>validation_pr_auc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:45</td>\n",
       "      <td>0.010 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.989796</td>\n",
       "      <td>4.584967</td>\n",
       "      <td>0.866007</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.989796</td>\n",
       "      <td>4.584967</td>\n",
       "      <td>0.867512</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:46</td>\n",
       "      <td>0.345 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.981584</td>\n",
       "      <td>4.064828</td>\n",
       "      <td>0.365268</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.981879</td>\n",
       "      <td>4.084868</td>\n",
       "      <td>0.386855</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:46</td>\n",
       "      <td>0.589 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.975317</td>\n",
       "      <td>3.822181</td>\n",
       "      <td>0.364095</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.975852</td>\n",
       "      <td>3.853578</td>\n",
       "      <td>0.387687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:46</td>\n",
       "      <td>0.852 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.969404</td>\n",
       "      <td>3.645022</td>\n",
       "      <td>0.363840</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.970166</td>\n",
       "      <td>3.685062</td>\n",
       "      <td>0.387479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:47</td>\n",
       "      <td>1.110 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.963683</td>\n",
       "      <td>3.503089</td>\n",
       "      <td>0.363432</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.964669</td>\n",
       "      <td>3.550449</td>\n",
       "      <td>0.387479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:47</td>\n",
       "      <td>1.399 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.958093</td>\n",
       "      <td>3.383978</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.959299</td>\n",
       "      <td>3.437732</td>\n",
       "      <td>0.388311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:47</td>\n",
       "      <td>1.692 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.952597</td>\n",
       "      <td>3.280713</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.954023</td>\n",
       "      <td>3.340154</td>\n",
       "      <td>0.388311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:47</td>\n",
       "      <td>2.032 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.947184</td>\n",
       "      <td>3.189588</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.948829</td>\n",
       "      <td>3.254264</td>\n",
       "      <td>0.388311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:48</td>\n",
       "      <td>2.358 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.941846</td>\n",
       "      <td>3.107945</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.943709</td>\n",
       "      <td>3.177402</td>\n",
       "      <td>0.388311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:48</td>\n",
       "      <td>2.691 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.936570</td>\n",
       "      <td>3.033831</td>\n",
       "      <td>0.363738</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.938650</td>\n",
       "      <td>3.107704</td>\n",
       "      <td>0.387895</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:48</td>\n",
       "      <td>2.985 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.931338</td>\n",
       "      <td>2.965584</td>\n",
       "      <td>0.363687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.933635</td>\n",
       "      <td>3.043692</td>\n",
       "      <td>0.387895</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:49</td>\n",
       "      <td>3.243 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.926156</td>\n",
       "      <td>2.902436</td>\n",
       "      <td>0.363687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.928669</td>\n",
       "      <td>2.984427</td>\n",
       "      <td>0.387895</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:49</td>\n",
       "      <td>3.495 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.921028</td>\n",
       "      <td>2.843769</td>\n",
       "      <td>0.363534</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.923761</td>\n",
       "      <td>2.929714</td>\n",
       "      <td>0.388311</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:49</td>\n",
       "      <td>3.756 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.915957</td>\n",
       "      <td>2.789250</td>\n",
       "      <td>0.363687</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.918907</td>\n",
       "      <td>2.878823</td>\n",
       "      <td>0.387895</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:49</td>\n",
       "      <td>4.012 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.910934</td>\n",
       "      <td>2.738108</td>\n",
       "      <td>0.363126</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.914103</td>\n",
       "      <td>2.831229</td>\n",
       "      <td>0.388103</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:51:54</td>\n",
       "      <td>8.125 sec</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.786695</td>\n",
       "      <td>1.924839</td>\n",
       "      <td>0.360424</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.796223</td>\n",
       "      <td>2.084962</td>\n",
       "      <td>0.389559</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2020-11-24 14:52:03</td>\n",
       "      <td>17.703 sec</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.655787</td>\n",
       "      <td>1.371758</td>\n",
       "      <td>0.352011</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.676651</td>\n",
       "      <td>1.607245</td>\n",
       "      <td>0.388935</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2020-11-24 14:51:45   0.010 sec              0.0       0.989796   \n",
       "1     2020-11-24 14:51:46   0.345 sec              1.0       0.981584   \n",
       "2     2020-11-24 14:51:46   0.589 sec              2.0       0.975317   \n",
       "3     2020-11-24 14:51:46   0.852 sec              3.0       0.969404   \n",
       "4     2020-11-24 14:51:47   1.110 sec              4.0       0.963683   \n",
       "5     2020-11-24 14:51:47   1.399 sec              5.0       0.958093   \n",
       "6     2020-11-24 14:51:47   1.692 sec              6.0       0.952597   \n",
       "7     2020-11-24 14:51:47   2.032 sec              7.0       0.947184   \n",
       "8     2020-11-24 14:51:48   2.358 sec              8.0       0.941846   \n",
       "9     2020-11-24 14:51:48   2.691 sec              9.0       0.936570   \n",
       "10    2020-11-24 14:51:48   2.985 sec             10.0       0.931338   \n",
       "11    2020-11-24 14:51:49   3.243 sec             11.0       0.926156   \n",
       "12    2020-11-24 14:51:49   3.495 sec             12.0       0.921028   \n",
       "13    2020-11-24 14:51:49   3.756 sec             13.0       0.915957   \n",
       "14    2020-11-24 14:51:49   4.012 sec             14.0       0.910934   \n",
       "15    2020-11-24 14:51:54   8.125 sec             43.0       0.786695   \n",
       "16    2020-11-24 14:52:03  17.703 sec            100.0       0.655787   \n",
       "\n",
       "    training_logloss  training_classification_error  training_auc  \\\n",
       "0           4.584967                       0.866007           NaN   \n",
       "1           4.064828                       0.365268           NaN   \n",
       "2           3.822181                       0.364095           NaN   \n",
       "3           3.645022                       0.363840           NaN   \n",
       "4           3.503089                       0.363432           NaN   \n",
       "5           3.383978                       0.363636           NaN   \n",
       "6           3.280713                       0.363636           NaN   \n",
       "7           3.189588                       0.363636           NaN   \n",
       "8           3.107945                       0.363636           NaN   \n",
       "9           3.033831                       0.363738           NaN   \n",
       "10          2.965584                       0.363687           NaN   \n",
       "11          2.902436                       0.363687           NaN   \n",
       "12          2.843769                       0.363534           NaN   \n",
       "13          2.789250                       0.363687           NaN   \n",
       "14          2.738108                       0.363126           NaN   \n",
       "15          1.924839                       0.360424           NaN   \n",
       "16          1.371758                       0.352011           NaN   \n",
       "\n",
       "    training_pr_auc  validation_rmse  validation_logloss  \\\n",
       "0               NaN         0.989796            4.584967   \n",
       "1               NaN         0.981879            4.084868   \n",
       "2               NaN         0.975852            3.853578   \n",
       "3               NaN         0.970166            3.685062   \n",
       "4               NaN         0.964669            3.550449   \n",
       "5               NaN         0.959299            3.437732   \n",
       "6               NaN         0.954023            3.340154   \n",
       "7               NaN         0.948829            3.254264   \n",
       "8               NaN         0.943709            3.177402   \n",
       "9               NaN         0.938650            3.107704   \n",
       "10              NaN         0.933635            3.043692   \n",
       "11              NaN         0.928669            2.984427   \n",
       "12              NaN         0.923761            2.929714   \n",
       "13              NaN         0.918907            2.878823   \n",
       "14              NaN         0.914103            2.831229   \n",
       "15              NaN         0.796223            2.084962   \n",
       "16              NaN         0.676651            1.607245   \n",
       "\n",
       "    validation_classification_error  validation_auc  validation_pr_auc  \n",
       "0                          0.867512             NaN                NaN  \n",
       "1                          0.386855             NaN                NaN  \n",
       "2                          0.387687             NaN                NaN  \n",
       "3                          0.387479             NaN                NaN  \n",
       "4                          0.387479             NaN                NaN  \n",
       "5                          0.388311             NaN                NaN  \n",
       "6                          0.388311             NaN                NaN  \n",
       "7                          0.388311             NaN                NaN  \n",
       "8                          0.388311             NaN                NaN  \n",
       "9                          0.387895             NaN                NaN  \n",
       "10                         0.387895             NaN                NaN  \n",
       "11                         0.387895             NaN                NaN  \n",
       "12                         0.388311             NaN                NaN  \n",
       "13                         0.387895             NaN                NaN  \n",
       "14                         0.388103             NaN                NaN  \n",
       "15                         0.389559             NaN                NaN  \n",
       "16                         0.388935             NaN                NaN  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Origin</td>\n",
       "      <td>156621.484375</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.505359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UniqueCarrier</td>\n",
       "      <td>144643.531250</td>\n",
       "      <td>0.923523</td>\n",
       "      <td>0.466710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fDayofMonth</td>\n",
       "      <td>4113.584473</td>\n",
       "      <td>0.026264</td>\n",
       "      <td>0.013273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fMonth</td>\n",
       "      <td>3239.199219</td>\n",
       "      <td>0.020682</td>\n",
       "      <td>0.010452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IsDepDelayed</td>\n",
       "      <td>964.104858</td>\n",
       "      <td>0.006156</td>\n",
       "      <td>0.003111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>fDayOfWeek</td>\n",
       "      <td>339.425415</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>0.001095</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        variable  relative_importance  scaled_importance  percentage\n",
       "0         Origin        156621.484375           1.000000    0.505359\n",
       "1  UniqueCarrier        144643.531250           0.923523    0.466710\n",
       "2    fDayofMonth          4113.584473           0.026264    0.013273\n",
       "3         fMonth          3239.199219           0.020682    0.010452\n",
       "4   IsDepDelayed           964.104858           0.006156    0.003111\n",
       "5     fDayOfWeek           339.425415           0.002167    0.001095"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# domain is too big\n",
    "\n",
    "air = h2o.import_file(path=_locate(\"smalldata/airlines/AirlinesTrain.csv.zip\"))\n",
    "\n",
    "features = [\"Origin\", \"Dest\", \"IsDepDelayed\", \"UniqueCarrier\", \"fMonth\", \"fDayofMonth\", \"fDayOfWeek\"]\n",
    "response = \"Dest\"\n",
    "\n",
    "r = air[0].runif()\n",
    "train = air[r < 0.8]\n",
    "valid = air[r >= 0.8]\n",
    "\n",
    "#Too many domains - AUC/PR AUC is not calculated\n",
    "gbm = H2OGradientBoostingEstimator(distribution=\"multinomial\", \n",
    "                                   ntrees=100, \n",
    "                                   max_depth=3, \n",
    "                                   learn_rate=0.01,\n",
    "                                   auc_type=\"MACRO_OVO\")\n",
    "gbm.train(x               =features, \n",
    "          y               =response, \n",
    "          training_frame  =train,\n",
    "          validation_frame=valid)\n",
    "gbm.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Table is not computed because it is disabled or due to domain size (maximum is 50 domains).'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm.multinomial_auc_table()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
